import requests,re
import concurrent
from concurrent.futures import ThreadPoolExecutor




# url = ['https://www.ranwen8.com/wanben/{}'.format(i) for i in range(1,50) ]
# print(url)
# for urla in url:
#     res = requests.get(urla)
#     res.encoding = 'gbk'
#     html = res.text
#     htmlmsg = re.findall(r'<td><a href="(.*?)" title=',html,re.S)
#     print(htmlmsg)

def validateTitle(title):
    rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(rstr, "_", title)  # 替换为下划线
    return new_title


url = 'https://www.ranwen8.com/book/117427.html'

# 获取一本书的所有章节的url+章节名
def get_chapter(url):
    res = requests.get(url)
    res.encoding = 'gbk'
    html = res.text
    htmlmsg = re.findall(r'<dd class="col-md-3"><a href="(.*?)" title=".*?">.*?</a></dd>',html,re.S)
    htmlmsg = ['https://www.ranwen8.com{}'.format(i) for i in htmlmsg]
    print(htmlmsg)
    return htmlmsg


# 获取一个章节的文字
def get_chapter_txt(url):
    # url ='https://www.ranwen8.com/book/117427/43852875.html'
    res = requests.get(url)
    res.encoding = 'gbk'
    html = res.text
    title = validateTitle(re.findall(r'<h1 class="readTitle">(.*?)<small></small></h1>', html, re.S)[0]).strip()
    print(title)

    txt = re.findall(r'<div class="panel-body" id="htmlContent">(.*?)</div>', html, re.S)[0]
    ad = re.findall(r'<div class="panel-body" id="htmlContent">(.*?)<br><br>', html, re.S)[0]
    # 清洗数据
    txt = txt.replace(ad,'').replace('&nbsp;','').replace('<br />','').replace('<br>','')
    print(txt)
    # 写入数据
    with open(title +'.txt', 'w+', encoding='gbk', errors='ignore') as f:
        f.write(title + '\n' + txt)

# 下载所有小说文字
def download_all_txt(urls):
    with concurrent.futures.ProcessPoolExecutor(max_workers=5) as exector:
        for url in urls:
            exector.submit(get_chapter_txt, url)



if __name__ == '__main__':
    urls = get_chapter(url)
    download_all_txt(urls)
